{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Draw\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAGdJJREFUeJzt3X20XXV95/H3hwSGRwnIg5GHiTOaDA6VWBgGRREIsgBZ\nRnF0YGknVKadOi0Fq7VYOyrL5RrqY52xo0MBYSqmUp5UOtVEDWZcAyjEBBISg0rAREigPEmdgpjv\n/LF/17lze+89Z//2vjffdfy81jrr7vPw+57PvXff3zl3n72/WxGBmZmNlt12dQAzM+ufJ3czsxHk\nyd3MbAR5cjczG0Ge3M3MRpAndzOzETRwcpd0hKRVku6VtEHSReX2AyWtlHRf+XrAzMc1M7NhaNB+\n7pLmA/MjYo2k/YC7gDcA5wOPRcRlki4BDoiIP5rpwGZmNtjAd+4R8VBErCnLPwU2AocBS4FrysOu\noZnwzcwsgYHv3P+/B0sLgNXA0cCDETGv3C7g8bHrE8b8NvDbAHvuueexRx55ZKfAO3fuZLfd6j8q\n6Dp+lGpkyJClRoYMWWpkyJClRoYMAJs3b340Ig5uNSgihroA+9JskjmnXH9iwv2PD6qxcOHC6GrV\nqlW7dPwo1ciQIUuNDBmy1MiQIUuNDBkiIoA7Y8i5euwy1MuJpN2BG4BrI+LGcvP2sj1+bLv8jlav\nKmZmNmOG2VtGwJXAxoj4xLi7vgwsK8vLgC/1H8/MzGrMHeIxJwK/AdwjaW257Y+By4DrJF0APAC8\nZWYimplZWwMn94j4NqAp7l7SbxwzM+uDj1A1MxtBntzNzEbQMB+oXiVph6T14247RtJtku6R9BVJ\nz5vZmGZm1sYw79yvBs6YcNsVwCUR8WvATcAf9pzLzMw6GKb9wGrgsQk3L6Q5UhVgJfCmnnOZmVkH\nQ7UfKG0HbomIo8v1/w18JCJulvQHwKURsd8UY3/ZfuDggw8+9rrrrusU+Omnn2bffffdZeNHqUaG\nDFlqZMiQpUaGDFlqZMgAcMopp9wVEce1GjTMYazAAmD9uOv/AlhB047gA8DfDVPH7Qdy1ciQIUuN\nDBmy1MiQIUuNDBki6toPDHMQ02QvCJuA0wEkLQReV1PHzMxmRtWukJIOKV93A/4E+GyfoczMrJth\ndoVcDtwGLJK0tbQbOE/SZmAT8BPgczMb08zM2him/cB5U9z1qZ6zmJlZT3yEqpnZCPLkbmY2gmrb\nDyyWdLuktZLulHT8zMY0M7M2atsPfITmwKXFwPvLdTMzS6K2/UAAY83C9qfZY8bMzJKobT9wFPA1\nmpN47Aa8MiIemGKs2w8krZEhQ5YaGTJkqZEhQ5YaGTLA7LYf+C/Am8ryW4CvD1PH7Qdy1ciQIUuN\nDBmy1MiQIUuNDBki6toP1O4tswy4sSz/NeAPVM3MEqmd3H8CvKYsnwrc108cMzPrw8AjVEv7gZOB\ngyRtpekC+VvApyTNBf6Bsk3dzMxy6NJ+4Nies5iZWU98hKqZ2Qjy5G5mNoJq2w98sbQeWCtpi6S1\nMxvTzMzaGOZMTFcDnwb+x9gNEfFvx5YlfRx4svdkZmZWbZgPVFeXI1T/EUmiOYjp1H5jmZlZF1Xt\nB8bdfhLwiZjmsFi3H8hbI0OGLDUyZMhSI0OGLDUyZIBZbD8w7vbPAO8a9nBYtx/IVSNDhiw1MmTI\nUiNDhiw1MmSIqGs/MMw290mVA5jOwfu7m5ml02VXyNOATRGxta8wZmbWj2F2hVwO3AYskrRV0gXl\nrnOB5TMZzszM6lS3H4iI83tPY2ZmvfARqmZmI8iTu5nZCKpqP1Buv1DSJkkbJPkE2WZmiQzzzv1q\n4IzxN0g6BVgKHBMR/xL4WP/RzMys1sDJPSJWA49NuPkdwGUR8Ux5zI4ZyGZmZpWq2g+ULpBfonlH\n/w/AuyPiu1OMdfuBpDUyZMhSI0OGLDUyZMhSI0MGmMX2A8B64L8Cojk59v2UF4rpLm4/kKtGhgxZ\namTIkKVGhgxZamTIEFHXfqB2b5mtwI3leb8D7AQOqqxlZmY9q53cbwZOAZC0ENgDeLSvUGZm1s3A\nI1RL+4GTgYMkbQU+AFwFXFV2j3wWWFb+dTAzswSq2w8Ab+s5i5mZ9cRHqJqZjSBP7mZmI6iq/YCk\nD0raJmltuZw1szHNzKyNqvYDxScjYnG5/M9+Y5mZWRe17QfMzCyx2vYDHwR+E3gSuJPmJNmPTzHW\n7QeS1siQIUuNDBmy1MiQIUuNDBlgdtsPHArMoXnn/2HgqmHquP1ArhoZMmSpkSFDlhoZMmSpkSFD\nxCy2H4iI7RHxi4jYCfwFTX8ZMzNLompylzR/3NU30jQSMzOzJGrbD5wsaTEQwBbgP8xgRjMza6m2\n/cCVM5DFzMx64iNUzcxGUPUJsst975IUktzL3cwskeojVCUdAZwOPNhzJjMz66jLEaqfBN5D86Gq\nmZklUrsr5FJgW0Ss6zmPmZn1oHX7AUl7A6uA0yPiSUlbgOMiYtLT7Ln9QN4aGTJkqZEhQ5YaGTJk\nqZEhA8xS+wHg14AdNPu3bwGeo9nu/oJBddx+IFeNDBmy1MiQIUuNDBmy1MiQIaKu/cDA/dwneTG4\nBzhk7Pqgd+5mZjb7htkVcjlwG7BI0lZJF8x8LDMz66LLCbLH7l/QWxozM+uFj1A1MxtBntzNzEZQ\n7QmyPyTp7nJy7BWSXjizMc3MrI3a9gMfjYiXRcRi4Bbg/X0HMzOzelXtByLiqXFX98EtCMzMUqk6\nQXa57cPAv6M5SfYpEfHIFGN9hGrSGhkyZKmRIUOWGhkyZKmRIQPM4gmyJ9z3XuDSYer4CNVcNTJk\nyFIjQ4YsNTJkyFIjQ4aIWTxB9gTXAm/qoY6ZmfWktivkS8ZdXQps6ieOmZn1ofYE2WdJWgTsBB4A\nfmcmQ5qZWTs+QbaZ2QjyEapmZiPIk7uZ2QiqbT/wUUmbSguCmyTNm9mYZmbWRm37gZXA0RHxMmAz\nzb7uZmaWRG37gRUR8Vy5ejtw+AxkMzOzStXtB8bd9xXgixHx+SnGuv1A0hoZMmSpkSFDlhoZMmSp\nkSED7IL2A8D7gJsoLxKDLm4/kKtGhgxZamTIkKVGhgxZamTIEDFLJ8geI+l84GxgSXlyMzNLompy\nl3QG8B7gNRHxs34jmZlZV8PsCrkcuA1YJGmrpAuATwP7ASvL2Zg+O8M5zcysBbcfMDMbQT5C1cxs\nBHlyNzMbQbXtB94saYOknZLa7XtpZmYzrrb9wHrgHGB134HMzKy7YT5QXV2OUB1/20YASTOTyszM\nOunUfkDSrcC7I+LOaca6/UDSGhkyZKmRIUOWGhkyZKmRIQPsmvYDtwLHDXs4rNsP5KqRIUOWGhky\nZKmRIUOWGhkyRNS1H/DeMmZmI8iTu5nZCKpqPyDpjZK2Aq8A/kbS12Y6qJmZDa+2/QA0rX7NzCwh\nb5YxMxtBntzNzEZQbfuBAyWtlHRf+XrAzMY0M7M2atsPXAJ8IyJeAnyjXDczsyQGTu4RsRp4bMLN\nS4FryvI1wBt6zmVmZh1UtR+Q9EREzCvLAh4fuz7JWLcfSFojQ4YsNTJkyFIjQ4YsNTJkgFlsPwA8\nMeH+x4ep4/YDuWpkyJClRoYMWWpkyJClRoYMEbPbfmC7pPkA5euOyjpmZjYDaif3LwPLyvIy4Ev9\nxDEzsz5UtR8ALgNeK+k+4LRy3czMkujSfmBJz1nMzKwnPkLVzGwEeXI3MxtBnSZ3SRdJWi9pg6SL\n+wplZmbdVE/uko4Gfgs4HjgGOFvSi/sKZmZm9bq8cz8KuCMifhYRzwHfAs7pJ5aZmXUxVPuBSQdK\nR9Hs3/4K4P/QNBC7MyIunPA4tx9IWiNDhiw1MmTIUiNDhiw1MmSAGWw/MNUFuAC4C1gNfAb4s+ke\n7/YDuWpkyJClRoYMWWpkyJClRoYMEbPbfmDsheHKiDg2Ik4CHgc2d6lnZmb9GHgQ03QkHRIROyQd\nSbO9/YR+YpmZWRedJnfgBknPB34O/G5EPNFDJjMz66jT5B4Rr+4riJmZ9cdHqJqZjSBP7mZmI6hr\n+4F3ltYD6yUtl7RnX8HMzKxel/YDhwG/DxwXzblV5wDn9hXMzMzqdd0sMxfYS9JcYG/gJ90jmZlZ\nV9XtB6DpCgl8mKb9wIqIeOskj3H7gaQ1MmTIUiNDhiw1MmTIUiNDBpjl9gPAAcA3gYOB3YGbgbdN\nN8btB3LVyJAhS40MGbLUyJAhS40MGSJmv/3AacD9EfFIRPwcuBF4ZYd6ZmbWky6T+4PACZL2liSa\nc6pu7CeWmZl1UT25R8QdwPXAGuCeUuvynnKZmVkHXdsPfAD4QE9ZzMysJz5C1cxsBHlyNzMbQV2O\nUF0kae24y1OSLu4znJmZ1ane5h4R3wcWA0iaA2wDbuopl5mZddDXZpklwA8j4oGe6pmZWQed2g/8\nsoh0FbAmIj49yX1uP5C0RoYMWWpkyJClRoYMWWpkyACz3H5g7ALsATwKHDrosW4/kKtGhgxZamTI\nkKVGhgxZamTIEDH77QfGnEnzrn17D7XMzKwHfUzu5wHLe6hjZmY96Xompn2A19I0DTMzsyS6th/4\ne+D5PWUxM7Oe+AhVM7MR1HWzzDxJ10vaJGmjpFf0FczMzOp12iwDfAr4akT8G0l70JxH1czMdrHq\nyV3S/sBJwPkAEfEs8Gw/sczMrIsum2VeBDwCfE7S9yRdUfaeMTOzXay6/YCk44DbgRMj4g5JnwKe\nioj/NOFxbj+QtEaGDFlqZMiQpUaGDFlqZMgAs9x+AHgBsGXc9VcDfzPdGLcfyFUjQ4YsNTJkyFIj\nQ4YsNTJkiJjl9gMR8TDwY0mLyk1LgHtr65mZWX+67i1zIXBt2VPmR8Bvdo9kZmZddT1CdS3QbjuQ\nmZnNOB+hamY2gjy5m5mNoE6bZSRtAX4K/AJ4LtruqmNmZjOi6weqAKdExKM91DEzs554s4yZ2Qjq\ndIJsSfcDT9JslvnvEXH5JI/xEapJa2TIkKVGhgxZamTIkKVGhgywC06QDRxWvh4CrANOmu7xPkI1\nV40MGbLUyJAhS40MGbLUyJAhYhecIDsitpWvO4CbgOO71DMzs35UT+6S9pG039gycDqwvq9gZmZW\nr8veMocCN0kaq/OFiPhqL6nMzKyT6sk9In4EHNNjFjMz64l3hTQzG0Ge3M3MRlDnyV3SnHKavVv6\nCGRmZt318c79ImBjD3XMzKwnnSZ3SYcDrwOu6CeOmZn1oWv7geuB/wzsB7w7Is6e5DFuP5C0RoYM\nWWpkyJClRoYMWWpkyACzf4Lss4H/VpZPBm4ZNMbtB3LVyJAhS40MGbLUyJAhS40MGSJmv/3AicDr\nS0/3vwJOlfT5DvXMzKwn1ZN7RLw3Ig6PiAXAucA3I+JtvSUzM7Nq3s/dzGwE9XEmJiLiVuDWPmqZ\nmVl3fuduZjaCPLmbmY2gLv3c95T0HUnrJG2QdGmfwczMrF6Xbe7PAKdGxNOSdge+LelvI+L2nrKZ\nmVmlLv3cA3i6XN29XOoPdzUzs950bT8wB7gLeDHw5xHxR5M8xu0HktbIkCFLjQwZstTIkCFLjQwZ\nYJbbD4y/APOAVcDR0z3O7Qdy1ciQIUuNDBmy1MiQIUuNDBkiZr/9wPgXiCfK5H5GH/XMzKybLnvL\nHCxpXlneC3gtsKmvYGZmVq/L3jLzgWvKdvfdgOsiwmdjMjNLoMveMncDL+8xi5mZ9cRHqJqZjSBP\n7mZmI6jLB6pHSFol6d7SfuCiPoOZmVm9Lh+oPge8KyLWSNoPuEvSyoi4t6dsZmZWqcuZmB6KiDVl\n+afARuCwvoKZmVm9Tu0HfllEWgCspjlC9akJ97n9QNIaGTJkqZEhQ5YaGTJkqZEhA+yi9gPAvjT9\nZc4Z9Fi3H8hVI0OGLDUyZMhSI0OGLDUyZIjYBe0HSqvfG4BrI+LGLrXMzKw/XfaWEXAlsDEiPtFf\nJDMz66rLO/cTgd8ATpW0tlzO6imXmZl10KX9wLcB9ZjFzMx64iNUzcxGkCd3M7MR1HVvmask7ZC0\nvq9AZmbWXdd37lfjsy+ZmaXTaXKPiNXAYz1lMTOznnRuP1BaD9wSEUdPcb/bDyStkSFDlhoZMmSp\nkSFDlhoZMsCuaz+wAFg/zGPdfiBXjQwZstTIkCFLjQwZstTIkCFiF7QfMDOznDy5m5mNoK67Qi4H\nbgMWSdoq6YJ+YpmZWRddzsRERJzXVxAzM+uPN8uYmY0gT+5mZiOo6zb3MyR9X9IPJF3SVygzM+um\ny8k65gB/DpwJvBQ4T9JL+wpmZmb1urxzPx74QUT8KCKeBf4KWNpPLDMz66LL3jKHAT8ed30r8K8n\nPmh8+wHgmR46SB4EPLoLx49SjQwZstTIkCFLjQwZstTIkAFgUdsBnXaFHEZEXA5cDiDpzmjbH2GC\nrjUyZMhSI0OGLDUyZMhSI0OGLDUyZBir0XZMl80y24Ajxl0/vNxmZma7WJfJ/bvASyS9SNIewLnA\nl/uJZWZmXXQ5QfZzkn4P+BowB7gqIjYMGHZ57fP1WCNDhiw1MmTIUiNDhiw1MmTIUiNDhqoanfu5\nm5lZPj5C1cxsBHlyNzMbQbMyuffRpkDSVZJ21O4nL+kISask3Stpg6SLKmrsKek7ktaVGpdWZpkj\n6XuSbqkcv0XSPZLW1uwiVWrMk3S9pE2SNkp6Rcvxi8rzj12eknRxyxrvLD/H9ZKWS9qz3XcBki4q\n4zcM+/yTrUuSDpS0UtJ95esBFTXeXHLslDRw17cpany0/E7ulnSTpHktx3+ojF0raYWkF7bNMO6+\nd0kKSQdVfB8flLRt3PpxVk0OSReWn8cGSR9pmeGL455/i6S1Fd/HYkm3j/2tSTq+osYxkm4rf7Nf\nkfS8acZPOk+1XT+B7qfZG3Sh+bD1h8A/A/YA1gEvrahzEvDrDHlKv0nGzwd+vSzvB2xumwMQsG9Z\n3h24AzihIssfAF+gOfdszfeyBTio4+/lGuDfl+U9gHkdf8cPA/+0xZjDgPuBvcr164DzWz7v0cB6\nYG+anQO+Dry4Zl0CPgJcUpYvAf60osZRNAeb3AocV5njdGBuWf7T6XJMMf5545Z/H/hs2wzl9iNo\ndpZ4YNC6NkWODwLvbvG7nKzGKeV3+k/K9UPafh/j7v848P6KDCuAM8vyWcCtFTW+C7ymLL8d+NA0\n4yedp9qunxGzc5q9XtoURMRq4LHaEBHxUESsKcs/BTbSTDBtakREPF2u7l4urT6RlnQ48Drgijbj\n+iRpf5qV8EqAiHg2Ip7oUHIJ8MOIeKDluLnAXpLm0kzQP2k5/ijgjoj4WUQ8B3wLOGfQoCnWpaU0\nL3iUr29oWyMiNkbE94fMPlWNFeV7Abid5viRNuOfGnd1Hwasn9P8XX0SeM+g8QNqDG2KGu8ALouI\nZ8pjdtRkkCTgLcDyigwBjL3T3p8B6+gUNRYCq8vySuBN04yfap5qtX7C7GyWmaxNQatJtW+SFgAv\np3nn3XbsnPLv3Q5gZUS0rfFnNH80O9s+9zgBfF3SXWraO7T1IuAR4HNl89AVkvbpkOdcBvzhTBQR\n24CPAQ8CDwFPRsSKls+7Hni1pOdL2pvmndURA8ZM5dCIeKgsPwwcWlmnT28H/rbtIEkflvRj4K3A\n+yvGLwW2RcS6tmMnuLBsIrpqqM0I/9hCmt/vHZK+JelfVeZ4NbA9Iu6rGHsx8NHy8/wY8N6KGhv4\nf29o38yQ6+iEear1+vkr94GqpH2BG4CLJ7zLGUpE/CIiFtO8ozpe0tEtnvtsYEdE3NX2eSd4Vclw\nJvC7kk5qOX4uzb+On4mIlwN/T/OvXmtqDmB7PfDXLccdQLPCvwh4IbCPpLe1qRERG2k2XawAvgqs\nBX7RpsYUdYOW/5H1TdL7gOeAa9uOjYj3RcQRZezvtXzevYE/puJFYYLP0GyKXUzz4v3xihpzgQOB\nE4A/BK4r78LbOo+Wbz7GeQfwzvLzfCflv92W3g78R0l30WxqeXbQgOnmqWHXz9mY3NO0KZC0O80P\n7NqIuLFLrbIZYxVwRothJwKvl7SFZvPUqZI+X/Hc28rXHcBNNJu+2tgKbB33X8f1NJN9jTOBNRGx\nveW404D7I+KRiPg5cCPwyrZPHhFXRsSxEXES8DjNNsoa2yXNByhfp9wEMNMknQ+cDby1/CHXupZp\nNgFM4Z/TvOCuK+vp4cAaSS9oUyQitpc3QjuBv6D9OgrNenpj2Rz6HZr/dqf9cHeissnvHOCLFc8P\nsIxm3YTmDUzr7yMiNkXE6RFxLM2LzA+ne/wU81Tr9XM2JvcUbQrKK/6VwMaI+ERljYPH9l6QtBfw\nWmDTsOMj4r0RcXhELKD5OXwzIlq9W5W0j6T9xpZpPoBrtQdRRDwM/FjSWKe5JcC9bWqMU/uu6EHg\nBEl7l9/NEprti61IOqR8PZLmj/gLFVmgWSeXleVlwJcq63Qi6QyazXavj4ifVYx/ybirS2mxfgJE\nxD0RcUhELCjr6VaaD/gebplj/rirb6TlOlrcTPOhKpIW0nzw37a74mnApojYWvH80Gxjf01ZPhVo\nvWln3Dq6G/AnwGeneexU81T79XPQJ659XGi2hW6mecV6X2WN5TT/3v2cZoW7oOX4V9H8K3M3zb/v\na4GzWtZ4GfC9UmM9Az59H1DrZCr2lqH5V3dduWzo8PNcDNxZvpebgQMqauwD/B2wf2WGS2kmn/XA\nX1L2imhZ43/RvDCtA5bUrkvA84Fv0Pzxfh04sKLGG8vyM8B24GsVNX5A8xnV2Do65d4uU4y/ofw8\n7wa+AhzWNsOE+7cweG+ZyXL8JXBPyfFlYH5FjT2Az5fvZw1watvvA7ga+J0O68WrgLvK+nUHcGxF\njYto5r/NwGWUzgBTjJ90nmq7fkaE2w+YmY2iX7kPVM3MfhV4cjczG0Ge3M3MRpAndzOzEeTJ3cxs\nBHlyNzMbQZ7czcxG0P8FbdwFh/Ad6DcAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1748f233e80>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From D:\\Miniconda\\Miniconda1\\lib\\site-packages\\tensorflow\\python\\util\\tf_should_use.py:170: initialize_all_variables (from tensorflow.python.ops.variables) is deprecated and will be removed after 2017-03-02.\n",
      "Instructions for updating:\n",
      "Use `tf.global_variables_initializer` instead.\n",
      "5000\n",
      "Loss: 26.0063\n",
      "Epsilon: 0.985260820207\n",
      "10000\n",
      "Loss: 8.47574\n",
      "Epsilon: 0.970741078213\n",
      "15000\n",
      "Loss: 5.58463\n",
      "Epsilon: 0.956437507015\n",
      "20000\n",
      "Loss: 2.54813\n",
      "Epsilon: 0.942346888248\n",
      "25000\n",
      "Loss: 0.610778\n",
      "Epsilon: 0.928466051465\n",
      "30000\n",
      "Loss: 0.43834\n",
      "Epsilon: 0.914791873419\n",
      "35000\n",
      "Loss: 0.0102898\n",
      "Epsilon: 0.90132127736\n",
      "40000\n",
      "Loss: 0.254961\n",
      "Epsilon: 0.88805123235\n",
      "45000\n",
      "Loss: 0.00123056\n",
      "Epsilon: 0.874978752571\n",
      "50000\n",
      "Loss: 0.00286637\n",
      "Epsilon: 0.862100896661\n",
      "55000\n",
      "Loss: 0.000682194\n",
      "Epsilon: 0.849414767047\n",
      "60000\n",
      "Loss: 0.995006\n",
      "Epsilon: 0.836917509297\n",
      "65000\n",
      "Loss: 0.00187536\n",
      "Epsilon: 0.824606311475\n",
      "70000\n",
      "Loss: 0.000307594\n",
      "Epsilon: 0.81247840351\n",
      "75000\n",
      "Loss: 0.000284887\n",
      "Epsilon: 0.800531056572\n",
      "75000\n",
      "Loss: 0.0317317\n",
      "Epsilon: 0.800531056572\n",
      "75000\n",
      "Loss: 0.0059803\n",
      "Epsilon: 0.800531056572\n",
      "80000\n",
      "Loss: 0.000125016\n",
      "Epsilon: 0.788761582456\n",
      "85000\n",
      "Loss: 1.12926e-06\n",
      "Epsilon: 0.777167332981\n",
      "90000\n",
      "Loss: 0.000552224\n",
      "Epsilon: 0.765745699393\n",
      "95000\n",
      "Loss: 6.19307e-06\n",
      "Epsilon: 0.754494111776\n",
      "100000\n",
      "Loss: 0.000113888\n",
      "Epsilon: 0.743410038475\n",
      "105000\n",
      "Loss: 1.25329e-06\n",
      "Epsilon: 0.732490985526\n",
      "110000\n",
      "Loss: 3.33131e-07\n",
      "Epsilon: 0.721734496098\n",
      "115000\n",
      "Loss: 7.83824e-08\n",
      "Epsilon: 0.711138149933\n",
      "120000\n",
      "Loss: 4.2477e-08\n",
      "Epsilon: 0.70069956281\n",
      "125000\n",
      "Loss: 1.64464e-06\n",
      "Epsilon: 0.690416386003\n",
      "125000\n",
      "Loss: 3.00791e-07\n",
      "Epsilon: 0.690416386003\n",
      "125000\n",
      "Loss: 3.13786e-07\n",
      "Epsilon: 0.690416386003\n",
      "125000\n",
      "Loss: 1.95467e-07\n",
      "Epsilon: 0.690416386003\n",
      "130000\n",
      "Loss: 2.21599e-07\n",
      "Epsilon: 0.680286305753\n",
      "135000\n",
      "Loss: 7.21146e-07\n",
      "Epsilon: 0.67030704275\n",
      "140000\n",
      "Loss: 1.19035e-09\n",
      "Epsilon: 0.660476351617\n",
      "145000\n",
      "Loss: 2.50293e-09\n",
      "Epsilon: 0.650792020407\n",
      "150000\n",
      "Loss: 1.19995e-08\n",
      "Epsilon: 0.641251870106\n",
      "155000\n",
      "Loss: 2.2247e-08\n",
      "Epsilon: 0.631853754138\n",
      "160000\n",
      "Loss: 9.89239e-09\n",
      "Epsilon: 0.622595557888\n",
      "165000\n",
      "Loss: 1.04774e-10\n",
      "Epsilon: 0.613475198223\n",
      "170000\n",
      "Loss: 1.25438e-09\n",
      "Epsilon: 0.604490623024\n",
      "175000\n",
      "Loss: 1.19326e-10\n",
      "Epsilon: 0.595639810723\n",
      "180000\n",
      "Loss: 1.65892e-10\n",
      "Epsilon: 0.58692076985\n",
      "185000\n",
      "Loss: 6.75771e-07\n",
      "Epsilon: 0.578331538584\n",
      "190000\n",
      "Loss: 2.32831e-10\n",
      "Epsilon: 0.569870184313\n",
      "190000\n",
      "Loss: 2.12458e-10\n",
      "Epsilon: 0.569870184313\n",
      "190000\n",
      "Loss: 1.0652e-09\n",
      "Epsilon: 0.569870184313\n",
      "190000\n",
      "Loss: 3.55067e-10\n",
      "Epsilon: 0.569870184313\n",
      "195000\n",
      "Loss: 7.85803e-11\n",
      "Epsilon: 0.561534803194\n",
      "200000\n",
      "Loss: 7.85803e-11\n",
      "Epsilon: 0.553323519733\n",
      "****************** Agent is in the State 0 **********************\n",
      "Step is  1 ****************** Agent is in the State 1 **********************\n",
      "Step is  2 ****************** Agent is in the State 2 **********************\n",
      "Step is  3 ****************** Agent is in the State 3 **********************\n",
      "Step is  4 ****************** Agent is in the State 4 **********************\n",
      "Step is  5 ****************** Agent is in the State 24 **********************\n",
      "Step is  6 ****************** Agent is in the State 25 **********************\n",
      "Step is  7 ****************** Agent is in the State 26 **********************\n",
      "Step is  8 ****************** Agent is in the State 46 **********************\n",
      "Step is  9 ****************** Agent is in the State 45 **********************\n",
      "Step is  10 ****************** Agent is in the State 25 **********************\n",
      "Step is  11 ****************** Agent is in the State 26 **********************\n",
      "Step is  12 ****************** Agent is in the State 46 **********************\n",
      "Step is  13 ****************** Agent is in the State 45 **********************\n",
      "Step is  14 ****************** Agent is in the State 25 **********************\n",
      "Step is  15 ****************** Agent is in the State 26 **********************\n",
      "Step is  16 ****************** Agent is in the State 46 **********************\n",
      "Step is  17 ****************** Agent is in the State 45 **********************\n",
      "Step is  18 ****************** Agent is in the State 25 **********************\n",
      "Step is  19 ****************** Agent is in the State 26 **********************\n",
      "Step is  20 ****************** Agent is in the State 46 **********************\n",
      "Step is  21 ****************** Agent is in the State 45 **********************\n",
      "Step is  22 ****************** Agent is in the State 25 **********************\n",
      "Step is  23 ****************** Agent is in the State 26 **********************\n",
      "Step is  24 ****************** Agent is in the State 46 **********************\n",
      "Step is  25 ****************** Agent is in the State 45 **********************\n",
      "Step is  26 ****************** Agent is in the State 25 **********************\n",
      "Step is  27 ****************** Agent is in the State 26 **********************\n",
      "Step is  28 ****************** Agent is in the State 46 **********************\n",
      "Step is  29 ****************** Agent is in the State 45 **********************\n",
      "Step is  30 ****************** Agent is in the State 25 **********************\n",
      "Step is  31 ****************** Agent is in the State 26 **********************\n",
      "Step is  32 ****************** Agent is in the State 46 **********************\n",
      "Step is  33 ****************** Agent is in the State 45 **********************\n",
      "Step is  34 ****************** Agent is in the State 25 **********************\n",
      "Step is  35 ****************** Agent is in the State 26 **********************\n",
      "Step is  36 ****************** Agent is in the State 46 **********************\n",
      "Step is  37 ****************** Agent is in the State 45 **********************\n",
      "Step is  38 ****************** Agent is in the State 25 **********************\n",
      "Step is  39 ****************** Agent is in the State 26 **********************\n",
      "Step is  40 ****************** Agent is in the State 46 **********************\n",
      "Step is  41 ****************** Agent is in the State 45 **********************\n",
      "Step is  42 ****************** Agent is in the State 25 **********************\n",
      "Step is  43 ****************** Agent is in the State 26 **********************\n",
      "Step is  44 ****************** Agent is in the State 46 **********************\n",
      "Step is  45 ****************** Agent is in the State 45 **********************\n",
      "Step is  46 ****************** Agent is in the State 25 **********************\n",
      "Step is  47 ****************** Agent is in the State 26 **********************\n",
      "Step is  48 ****************** Agent is in the State 46 **********************\n",
      "Step is  49 ****************** Agent is in the State 45 **********************\n",
      "Step is  50 ****************** Agent is in the State 25 **********************\n",
      "Step is  51 ****************** Agent is in the State 26 **********************\n",
      "Step is  52 ****************** Agent is in the State 46 **********************\n",
      "Step is  53 ****************** Agent is in the State 45 **********************\n",
      "Step is  54 ****************** Agent is in the State 25 **********************\n",
      "Step is  55 ****************** Agent is in the State 26 **********************\n",
      "Step is  56 ****************** Agent is in the State 46 **********************\n",
      "Step is  57 ****************** Agent is in the State 45 **********************\n",
      "Step is  58 ****************** Agent is in the State 25 **********************\n",
      "Step is  59 ****************** Agent is in the State 26 **********************\n",
      "Step is  60 ****************** Agent is in the State 46 **********************\n",
      "Step is  61 ****************** Agent is in the State 45 **********************\n",
      "Step is  62 ****************** Agent is in the State 25 **********************\n",
      "Step is  63 ****************** Agent is in the State 26 **********************\n",
      "Step is  64 ****************** Agent is in the State 46 **********************\n",
      "Step is  65 ****************** Agent is in the State 45 **********************\n",
      "Step is  66 ****************** Agent is in the State 25 **********************\n",
      "Step is  67 ****************** Agent is in the State 26 **********************\n",
      "Step is  68 ****************** Agent is in the State 46 **********************\n",
      "Step is  69 ****************** Agent is in the State 45 **********************\n",
      "Step is  70 ****************** Agent is in the State 25 **********************\n",
      "Step is  71 ****************** Agent is in the State 26 **********************\n",
      "Step is  72 ****************** Agent is in the State 46 **********************\n",
      "Step is  73 ****************** Agent is in the State 45 **********************\n",
      "Step is  74 ****************** Agent is in the State 25 **********************\n",
      "Step is  75 ****************** Agent is in the State 26 **********************\n",
      "Step is  76 ****************** Agent is in the State 46 **********************\n",
      "Step is  77 ****************** Agent is in the State 45 **********************\n",
      "Step is  78 ****************** Agent is in the State 25 **********************\n",
      "Step is  79 ****************** Agent is in the State 26 **********************\n",
      "Step is  80 ****************** Agent is in the State 46 **********************\n",
      "Step is  81 ****************** Agent is in the State 45 **********************\n",
      "Step is  82 ****************** Agent is in the State 25 **********************\n",
      "Step is  83 ****************** Agent is in the State 26 **********************\n",
      "Step is  84 ****************** Agent is in the State 46 **********************\n",
      "Step is  85 ****************** Agent is in the State 45 **********************\n",
      "Step is  86 ****************** Agent is in the State 25 **********************\n",
      "Step is  87 ****************** Agent is in the State 26 **********************\n",
      "Step is  88 ****************** Agent is in the State 46 **********************\n",
      "Step is  89 ****************** Agent is in the State 45 **********************\n",
      "Step is  90 ****************** Agent is in the State 25 **********************\n",
      "Step is  91 ****************** Agent is in the State 26 **********************\n",
      "Step is  92 ****************** Agent is in the State 46 **********************\n",
      "Step is  93 ****************** Agent is in the State 45 **********************\n",
      "Step is  94 ****************** Agent is in the State 25 **********************\n",
      "Step is  95 ****************** Agent is in the State 26 **********************\n",
      "Step is  96 ****************** Agent is in the State 46 **********************\n",
      "Step is  97 ****************** Agent is in the State 45 **********************\n",
      "Step is  98 ****************** Agent is in the State 25 **********************\n",
      "Step is  99 ****************** Agent is in the State 26 **********************\n",
      "Step is  100 ****************** Agent is in the State 46 **********************\n",
      "Step is  101 ****************** Agent is in the State 45 **********************\n"
     ]
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "from collections import deque\n",
    "import random\n",
    "import matplotlib.pyplot as plt\n",
    "class Q_Network:\n",
    "    N_Mesh=20\n",
    "    State_Number=N_Mesh*N_Mesh;\n",
    "    Action_Number=4;\n",
    "    Action_List=np.identity(Action_Number)\n",
    "    State_List=np.identity(State_Number)\n",
    "    Epsilon=0\n",
    "    Epsilon_Begin=1\n",
    "    Epsilon_Final=0.01\n",
    "    decay_rate=0.000003\n",
    "    Step=0\n",
    "    Batch_Number=20\n",
    "    #Explore = 100000.\n",
    "    Observe=1000#The number of steps of observation before the beginning of the training\n",
    "    Store_Memory = deque()\n",
    "    Cost_History=[]\n",
    "    Memory_Size = 5000\n",
    "    #Obstacle=np.random.randint(N_Mesh,State_Number-2,size=100)\n",
    "    #Obstacle=[20,3,5,16,21,23,27,29,31,33,34,56,57,66,67,78,81,86]\n",
    "    Obstacle=[]\n",
    "    Play_Done=None\n",
    "    def __init__(self,Learning_Rate=0.001,Gamma=0.9,Memory_Size=5000):\n",
    "        self.Learning_Rate=Learning_Rate\n",
    "        self.Gamma=Gamma\n",
    "        self.Memory_Size=Memory_Size\n",
    "        self.Draw_Obstacle()\n",
    "        self.New_Network()\n",
    "        self.session=tf.InteractiveSession()\n",
    "        self.session.run(tf.initialize_all_variables())\n",
    "     #***************************************************************\n",
    "    def New_Network(self):\n",
    "         #Construct a neural network\n",
    "        self.State_Input=tf.placeholder(tf.float32,[None,self.State_Number])#Input_Number=100, M=Batch*100\n",
    "        self.Action_Input=tf.placeholder(tf.float32,[None,self.Action_Number])#Input_Number=4, M=Batch*4\n",
    "        self.Q_Target_OI=tf.placeholder(dtype=tf.float32,shape=[None])\n",
    "        #Layer1\n",
    "        Layer1_Number=100\n",
    "        w1=tf.Variable(tf.random_normal([self.State_Number,Layer1_Number]))\n",
    "        b1=tf.Variable(tf.zeros([1,Layer1_Number])+0.1)\n",
    "        l1=tf.nn.relu(tf.matmul(self.State_Input,w1)+b1) #b1 use the propagation mechanism\n",
    "#         #Layer2\n",
    "#         Layer2_Number=10\n",
    "#         w2=tf.Variable(tf.random_normal([Layer1_Number,Layer2_Number]))\n",
    "#         b2=tf.Variable(tf.zeros([1,Layer2_Number]))\n",
    "#         l2=tf.nn.relu(tf.matmul(l1,w2)+b2)\n",
    "        #Layer3\n",
    "        Layer3_Number=self.Action_Number\n",
    "        w3=tf.Variable(tf.random_normal([Layer1_Number,Layer3_Number]))\n",
    "        b3=tf.Variable(tf.zeros([1,Layer3_Number]))\n",
    "        self.l3=tf.matmul(l1,w3)+b3#At this time, not yet use the active action, l3 is the output matrice, M=Batch*4\n",
    "        #self.l3=tf.tanh(tf.matmul(l1,w2)+b2)\n",
    "\n",
    "        self.Q_Value=tf.reduce_sum(tf.multiply(self.l3,self.Action_Input),reduction_indices=1)\n",
    "        self.Loss=tf.reduce_mean(tf.square(self.Q_Value-self.Q_Target_OI))\n",
    "        #self.Loss=tf.reduce_mean(tf.square(self.Q_Value-tf.tanh(self.Q_Target_OI)))\n",
    "        self.Optimizer=tf.train.GradientDescentOptimizer(self.Learning_Rate).minimize(self.Loss)\n",
    "        #Network's Output is like this form [19,20,297,30], argmax is 2, then choose Action2  \n",
    "        self.Predict=tf.argmax(self.l3,1)\n",
    "     #***************************************************************\n",
    "    def Select_Action(self,State_Index):\n",
    "        Current_State=self.State_List[State_Index:State_Index+1] #By example, Action_Index is 2, Current_State=[0,0,1,0， ，]\n",
    "        if np.random.uniform()<self.Epsilon:\n",
    "            Choose_Action_Index=np.random.randint(0,self.Action_Number)\n",
    "        else:\n",
    "            Action_QValue_Output=self.session.run(self.l3,feed_dict={self.State_Input:Current_State})\n",
    "            Choose_Action_Index=np.argmax(Action_QValue_Output)\n",
    "        #The first inegality means the train has beginned\n",
    "        #The second inegality means Epsilon has not yet decreased to final value\n",
    "        self.Epsilon=self.Epsilon_Final+(self.Epsilon_Begin-self.Epsilon_Final)*np.exp(-self.decay_rate*self.Step)\n",
    "#         if(self.Step%1000==0):\n",
    "#             print(\"epsilon \",self.Epsilon)\n",
    "        return Choose_Action_Index\n",
    "     #***************************************************************\n",
    "    def Next_State(self,Action_Index,State_Index):\n",
    "        State=State_Index+1\n",
    "        done=False\n",
    "        #Execute left motion\n",
    "        if(Action_Index==0):\n",
    "            if(State%self.N_Mesh==1):\n",
    "                State=State\n",
    "                R=-1000\n",
    "            elif((State+1) in self.Obstacle):#When encounter obstables, we stay \n",
    "                State=State\n",
    "                R=-1000\n",
    "            else:\n",
    "                State=State-1\n",
    "                R=0\n",
    "        elif(Action_Index==1):\n",
    "            if(State==self.State_Number-1):\n",
    "                State=State+1\n",
    "                done=True\n",
    "                R=100\n",
    "            elif(State%self.N_Mesh==0):\n",
    "                State=State\n",
    "                R=-100\n",
    "            elif((State+1) in self.Obstacle):\n",
    "                State=State\n",
    "                R=-100\n",
    "            else:\n",
    "                State=State+1\n",
    "                R=0.1\n",
    "        elif(Action_Index==2):\n",
    "            if(State==self.State_Number-self.N_Mesh):\n",
    "                State=State+self.N_Mesh\n",
    "                done=True\n",
    "                R=100\n",
    "            elif(self.State_Number+1-self.N_Mesh<=State<=self.State_Number-1):\n",
    "                State=State\n",
    "                R=-100\n",
    "            elif((State+self.N_Mesh) in self.Obstacle):\n",
    "                State=State+self.N_Mesh\n",
    "                R=-100\n",
    "            else:\n",
    "                State=State+self.N_Mesh\n",
    "                R=0.1\n",
    "        else:\n",
    "            if(0<=State<=self.N_Mesh):\n",
    "                State=State\n",
    "                R=-100\n",
    "            elif((State-self.N_Mesh) in self.Obstacle):\n",
    "                State=State-self.N_Mesh\n",
    "                R=-100\n",
    "            else:\n",
    "                R=0\n",
    "                State=State-self.N_Mesh\n",
    "        return State-1,R,done\n",
    "    \n",
    "    def Save_Memory(self,CURRENT_STATET,CHOOSE_ACTION,NEXT_STATE,REWARD,DONE):\n",
    "        Memory_Current_State=self.State_List[CURRENT_STATET:CURRENT_STATET+1]\n",
    "        Memory_Choose_Action=self.Action_List[CHOOSE_ACTION:CHOOSE_ACTION+1]\n",
    "        Memory_Next_State=self.State_List[NEXT_STATE:NEXT_STATE+1]\n",
    "        self.Store_Memory.append((Memory_Current_State,Memory_Choose_Action,Memory_Next_State,REWARD,DONE))\n",
    "        if len(self.Store_Memory)>self.Memory_Size:\n",
    "            self.Store_Memory.popleft()\n",
    "    \n",
    "     #***************************************************************\n",
    "    def Experience_Replay(self):\n",
    "        Batch=self.Batch_Number\n",
    "        MiniBatch=random.sample(self.Store_Memory,Batch)\n",
    "        Batch_Current_State = None\n",
    "        Batch_Execute_Action = None\n",
    "        Batch_Reward = None\n",
    "        Batch_Next_State = None\n",
    "        Batch_Done = None\n",
    "        \n",
    "        for Index in range(Batch):\n",
    "            if Batch_Current_State is None:\n",
    "                Batch_Current_State=MiniBatch[Index][0]\n",
    "            elif Batch_Current_State is not None:\n",
    "                Batch_Current_State=np.vstack((Batch_Current_State, MiniBatch[Index][0]))\n",
    "            #---------------------------------------------------------------------------------------   \n",
    "            #print(\"BEA\",MiniBatch[Index][1].shape)\n",
    "            if Batch_Execute_Action is None:\n",
    "                Batch_Execute_Action=MiniBatch[Index][1]\n",
    "            elif Batch_Execute_Action is not None:\n",
    "                Batch_Execute_Action=np.vstack((Batch_Execute_Action,MiniBatch[Index][1]))\n",
    "           #---------------------------------------------------------------------------------------           \n",
    "            if Batch_Reward is None:\n",
    "                Batch_Reward=MiniBatch[Index][3]\n",
    "            elif Batch_Reward is not None:\n",
    "                Batch_Reward=np.vstack((Batch_Reward,MiniBatch[Index][3]))\n",
    "            #---------------------------------------------------------------------------------------  \n",
    "            if Batch_Next_State is None:\n",
    "                Batch_Next_State=MiniBatch[Index][2]\n",
    "            elif Batch_Next_State is not None:\n",
    "                Batch_Next_State=np.vstack((Batch_Next_State,MiniBatch[Index][2]))\n",
    "            #---------------------------------------------------------------------------------------      \n",
    "            if Batch_Done is None:\n",
    "                Batch_Done=MiniBatch[Index][4]\n",
    "            elif Batch_Done is not None:\n",
    "                Batch_Done=np.vstack((Batch_Done,MiniBatch[Index][4]))\n",
    "            #Calculate the Q Value of the next State \n",
    "        Q_Next=self.session.run(self.l3,feed_dict={self.State_Input:Batch_Next_State})\n",
    "        Q_Target=[]\n",
    "        for i in range(Batch):\n",
    "            Each_Reward=Batch_Reward[i][0]#This is a 2D array because of the vstack\n",
    "            #Calculate the Target-Q-Value of each element in the Batch\n",
    "            Each_QValue=Each_Reward+self.Gamma*np.max(Q_Next[i]) #The network ouput has its own []\n",
    "            if Each_Reward<0:\n",
    "                Q_Target.append(Each_Reward)\n",
    "            else:\n",
    "                Q_Target.append(Each_QValue)\n",
    "        #print(self.session.run(self.Q_Value,feed_dict={self.Q_Target_OI:Q_Target}))\n",
    "        _,Cost,Rew=self.session.run([self.Q_Value,self.Loss,self.Optimizer],feed_dict={self.State_Input:Batch_Current_State,\n",
    "                                                                                        self.Action_Input:Batch_Execute_Action,\n",
    "                                                                                        self.Q_Target_OI: Q_Target})\n",
    "        self.Cost_History.append(Cost)\n",
    "        if self.Step%5000==0:\n",
    "            print(self.Step)  \n",
    "            print(\"Loss:\", Cost)  \n",
    "            print(\"Epsilon:\", self.Epsilon)   \n",
    "    #**************************************************************\n",
    "    def Train(self):\n",
    "        while True:\n",
    "            Train_Current_State=np.random.randint(0,self.State_Number-1)\n",
    "            if((Train_Current_State in self.Obstacle)is not True):\n",
    "                break\n",
    "        self.Epsilon = self.Epsilon_Begin\n",
    "        while True:\n",
    "            Train_Action=self.Select_Action(Train_Current_State)\n",
    "            #print(\"TA\",Train_Action)\n",
    "            Train_Next_State,Train_Reward,Train_Done=self.Next_State(Train_Action,Train_Current_State)\n",
    "            self.Save_Memory(Train_Current_State,Train_Action,Train_Next_State,Train_Reward,Train_Done)\n",
    "            if self.Step>self.Observe:\n",
    "                self.Experience_Replay()\n",
    "            if self.Step>200000:\n",
    "                 break;\n",
    "            if Train_Done:\n",
    "                while True:\n",
    "                    if((Train_Current_State in self.Obstacle)is not True):\n",
    "                        break\n",
    "            else:\n",
    "                Train_Current_State=Train_Next_State\n",
    "                self.Step+=1          \n",
    "    #***************************************************************\n",
    "    def Play(self):\n",
    "        self.Train()\n",
    "        Start_Room_Index=0;\n",
    "        Play_Current_State_Index=0\n",
    "        Play_Step=0;\n",
    "        print(\"****************** Agent is in the State\",Start_Room_Index,\"**********************\")\n",
    "        while(Play_Current_State_Index!=self.State_Number-1):\n",
    "            Play_Current_State=self.State_List[Play_Current_State_Index:Play_Current_State_Index+1]\n",
    "            Play_Action=(self.session.run(self.Predict,feed_dict={self.State_Input:Play_Current_State}))\n",
    "            if Play_Action==0:\n",
    "                Play_Current_State_Index=Play_Current_State_Index-1\n",
    "            elif Play_Action==1:\n",
    "                Play_Current_State_Index=Play_Current_State_Index+1\n",
    "            elif Play_Action==2:\n",
    "                Play_Current_State_Index=Play_Current_State_Index+self.N_Mesh\n",
    "            else:\n",
    "                Play_Current_State_Index=Play_Current_State_Index-self.N_Mesh\n",
    "            Play_Step+=1\n",
    "            print(\"Step is \",Play_Step,\"****************** Agent is in the State\", Play_Current_State_Index,\"**********************\")\n",
    "            if(Play_Step>100):\n",
    "                break\n",
    "        if Play_Current_State_Index==self.State_Number-1:\n",
    "            self.Graph()\n",
    "    #***************************************************************        \n",
    "    def Caculate_coordinate(self,State):\n",
    "        X_Axis=0.5+(State-1)%self.N_Mesh\n",
    "        Y_Axis=0.5+((State-1)//self.N_Mesh)\n",
    "        return X_Axis,Y_Axis \n",
    "    #***************************************************************  \n",
    "    def Graph(self): \n",
    "        Graph_Step=0\n",
    "        fig=plt.figure()\n",
    "        ax=fig.gca()\n",
    "        ax.set(xlim=[0, self.N_Mesh], ylim=[0, self.N_Mesh])\n",
    "        ax.set_xticks(np.arange(0,(self.N_Mesh+1)))\n",
    "        ax.set_yticks(np.arange(0,(self.N_Mesh+1)))\n",
    "        plt.grid()\n",
    "        Graph_Start_Index=0\n",
    "        Graph_Current_Index=Graph_Start_Index\n",
    "        while(Graph_Current_Index!=self.State_Number-1):\n",
    "            Graph_Current_State=self.State_List[Graph_Current_Index:Graph_Current_Index+1]\n",
    "            Graph_Action=(self.session.run(self.Predict,feed_dict={self.State_Input:Graph_Current_State}))\n",
    "            if(Graph_Action==0):\n",
    "                Graph_Next_Index=Graph_Current_Index-1\n",
    "            elif(Graph_Action==1):\n",
    "                Graph_Next_Index=Graph_Current_Index+1\n",
    "            elif(Graph_Action==2):\n",
    "                 Graph_Next_Index=Graph_Current_Index+self.N_Mesh\n",
    "            else:\n",
    "                 Graph_Next_Index=Graph_Current_Index-self.N_Mesh\n",
    "            Graph_Step+=1\n",
    "            X_State,Y_State=self.Caculate_coordinate(Graph_Current_Index+1)\n",
    "            X_Next_State,Y_Next_State=self.Caculate_coordinate(Graph_Next_Index+1)\n",
    "            if(Graph_Step==0):\n",
    "                plt.scatter(X_State,Y_State)  \n",
    "            else:\n",
    "                plt.scatter(X_State,Y_State) \n",
    "                plt.scatter(X_Next_State,Y_Next_State)\n",
    "                plt.plot([X_State,X_Next_State],[Y_State,Y_Next_State])\n",
    "            for J in np.arange(len(self.Obstacle)):\n",
    "                plt.scatter((self.Caculate_coordinate(self.Obstacle[J]+1))[0],(self.Caculate_coordinate(self.Obstacle[J]+1))[1],marker=\"x\")\n",
    "            Graph_Current_Index=Graph_Next_Index\n",
    "    plt.show()\n",
    "    def Draw_Obstacle(self):\n",
    "        print('Draw')\n",
    "        fig=plt.figure()\n",
    "        ax=fig.gca()\n",
    "        ax.set(xlim=[0, self.N_Mesh], ylim=[0, self.N_Mesh])\n",
    "        ax.set_xticks(np.arange(0,(self.N_Mesh+1)))\n",
    "        ax.set_yticks(np.arange(0,(self.N_Mesh+1)))\n",
    "        plt.grid()\n",
    "        for J in np.arange(len(self.Obstacle)):\n",
    "                plt.scatter((self.Caculate_coordinate(self.Obstacle[J]+1))[0],(self.Caculate_coordinate(self.Obstacle[J]+1))[1],marker=\"x\")\n",
    "        plt.show()\n",
    "if __name__ == \"__main__\":\n",
    "    q_network = Q_Network()\n",
    "    q_network.Play()\n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 0  1  2  3  4]\n",
      " [ 5  6  7  8  9]\n",
      " [10 11 12 13 14]\n",
      " [15 16 17 18 19]]\n",
      "[4 4 4 4]\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "H=np.arange(20).reshape(4,5)\n",
    "print(H)\n",
    "D=np.argmax(H,axis=1)\n",
    "print(D)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
